import java.io.BufferedReader;
import java.io.BufferedWriter;
import java.io.File;
import java.io.FileWriter;
import java.io.IOException;
import java.io.InputStreamReader;
import java.net.HttpURLConnection;
import java.net.URL;
import java.util.LinkedList;

public class crawler {

	static LinkedList<String> newURL = new LinkedList<String>();// 存放未读取网页
	static LinkedList<String> oldURL = new LinkedList<String>();// 存放已读取网页
	static LinkedList<String> title_link = new LinkedList<String>();
	static int i = 0;
	static int num_all = 0;
	static int num_link = 1;

	public static void main(String[] args) {
		// String url = "https://0000av.co/vod/11934.html";

		newURL.add("https://0000av.co/");
		// oldURL.add("https://0000av.co/vod/index.html");
		while (!newURL.isEmpty()) {
			// System.out.println("进入 " + newURL.getFirst());
			Get_Html(newURL.getFirst());

			oldURL.add(newURL.getFirst());
			newURL.remove(0);

			try {
				Thread.currentThread().sleep(1000);
			} catch (InterruptedException e) {
				// TODO Auto-generated catch block
				e.printStackTrace();
			}
		}
		System.out.println("没有网页了~");
	}

	// 传入新网址，获取title和链接存到link.txt中，并且在遇到新的链接时检测是否符合格式，是否在old链表中，选择是否存入newURL
	public static void Get_Html(String url2) {

		// System.out.println("进入Get_html方法中了");
		try {
			URL url = new URL(url2);
			url.openConnection();
			HttpURLConnection connection = (HttpURLConnection) url.openConnection();
			connection.setRequestProperty("User-Agent", "Mozilla/4.0 (compatible; MSIE 5.0; Windows NT; DigExt)");

			// System.out.println("在new in 之前");
			BufferedReader in = new BufferedReader(new InputStreamReader(connection.getInputStream(), "UTF-8"));
			// System.out.println("在new in 之后");

			String urlString = "";
			String current;
			while ((current = in.readLine()) != null) {
				urlString += current;
			}
			// System.out.println(urlString);

			String a = "onclick=\"this.focus();this.select()\" autocomplete=\"on\">";
			String b = "</textarea></div><div class=\"play-list\">";
			String href_head = "href=\"/vod/";
			String href_end = "\" title";
		

			if (urlString.indexOf(href_head) > 0 && urlString.indexOf(href_end) > 0) {
				String href = "";
				String temp = urlString;
				while (temp.indexOf(href_head) > 0) {

					// System.out.print(temp.indexOf(href_head)+href_head.length()+"
					// ");
					// System.out.print(temp.indexOf(href_end)+" ");
					// System.out.println(temp.indexOf(href_end)-(temp.indexOf(href_head)+href_head.length()));

					if ((temp.indexOf(href_head) + href_head.length()) > temp.indexOf(href_end)) {
						temp = temp.substring(temp.indexOf(href_end) + href_end.length(), temp.length());
						// System.out.println("跳过一个混蛋！");
						continue;
					}

					href = temp.substring(temp.indexOf(href_head) + href_head.length(),
							temp.indexOf(href_head) + href_head.length() + 20);
					if (href.indexOf(href_end) < 0)
						try{
						//href = href.substring(0, href.indexOf("\" target="));
						href = href.substring(0, href.indexOf(".html"))+".html";
						}catch(Exception e){
							System.out.println(e.getMessage());
							
						}
					else
						href = href.substring(0, href.indexOf(href_end));
					String link = "https://0000av.co/vod/" + href;

					temp = temp.substring(temp.indexOf(href) + href.length() + href_end.length(), temp.length());
					// System.out.println(temp);
					// System.out.println("发现网页 ： "+link);
					if (!newURL.contains(link)) {
						newURL.add(link);
						System.out.println("新增网页 ： " + link);
					}
				}
			}

			if (urlString.indexOf(a) > 0) {
				String link = urlString.substring(urlString.indexOf(a) + a.length(), urlString.indexOf(b));

				String title = urlString.substring(urlString.indexOf("<title>") + "<title>".length(),
						urlString.indexOf("-千百撸影院</title>"));

				if (title_link.contains(title))
					return;
				else
					title_link.add(title);
				System.out.println("【" + num_all + "】 " + title);// 片子标题
				System.out.println(link);// 迅雷链接
				File file = new File("link" + i + ".txt");
				if (!file.exists()) {
					file.createNewFile();
				}

				// true = append file
				FileWriter fileWritter = new FileWriter(file.getName(), true);
				BufferedWriter bufferWritter = new BufferedWriter(fileWritter);
				bufferWritter.write(title + " : ");
				bufferWritter.newLine();
				bufferWritter.write(link);
				bufferWritter.newLine();
				bufferWritter.newLine();
				bufferWritter.close();
				num_link++;
				num_all++;
				if (num_link > 100) {
					i++;
					num_link = 0;
				}
				// System.out.println("Done");
			} else
				System.out.println("此网页无下载链接");
			
			String href_list = "<a href=\"/list/";
			String temp2 = urlString;
			while (temp2.contains(href_list)) {

				temp2 = temp2.substring(temp2.indexOf(href_list) + href_list.length(), temp2.length());
				String temp_link = temp2.substring(0, temp2.indexOf(".html"));
				//System.out.println("temp2.indexof(href_list)="+temp2.indexOf(href_list) +"temp2.length = "+temp2.length());
				temp_link = "https://www.0000av.co/list/" + temp_link + ".html";
				if (!newURL.contains(temp_link)) {
					newURL.add(temp_link);
					System.out.println("新增网页 ： " + temp_link);
				}
			}
		} catch (IOException e) {
			e.getStackTrace();
		}
		// System.out.println("离开Get_html方法了");
	}

}
